import requests
from lxml import etree
import os

# 页面的rul
html_urls = []
for i in range(0, 5):
    i = i*20
    temp_html_url = 'https://movie.douban.com/review/best/?start={}'.format(i)
    html_urls.append(temp_html_url)

# 每一个页面中影评的url
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'}
file_path = os.getcwd() + os.sep + 'p06_douban_film_review.txt'
if os.path.exists(file_path):
    os.remove(file_path)
for temp_html_url in html_urls:
    html_response = requests.get(temp_html_url, headers=headers)
    html_content = html_response.content.decode('utf8')
    html = etree.HTML(html_content)
    review_urls = html.xpath('//div[@class="main-bd"]/h2/a/@href')
    for review_url in review_urls:
        try:
            review_html_response = requests.get(review_url, headers=headers)
            review_html_content = review_html_response.content.decode('utf8')
            review_html = etree.HTML(review_html_content)
            # 电影名
            movie_name = review_html.xpath('//div[@class="subject-title"]/a/text()')[0][2:]
            # 评论人和评分
            reviewer = review_html.xpath('//header/a/span/text()')[0]
            review_score = review_html.xpath('//header/span[1]/@title')[0]
            # 评论内容
            review_content_list = review_html.xpath('//div[@id="link-report"]/div/p//text()')
            with open(file_path,'a', encoding='utf8') as f:
                f.write('url:%s\n' % review_url)
                f.write('电影名：%s\n' % movie_name)
                f.write('评论人：%s\n' % reviewer)
                f.write('评分：%s\n' % review_score)
                f.write('品论：\n')
                for review_content in review_content_list:
                    f.write('%s\n' % review_content)
                f.write('*'*50)
                f.write('\n')
            print('爬取成功：%s' % review_url)
        except Exception:
            print('失败：%s' % review_url)
            with open(file_path, 'a', encoding='utf8') as f:
                f.write('url:%s\n' % review_url)
                f.write('爬取失败\n')
